{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf \n",
    "import os\n",
    "import time \n",
    "import datetime\n",
    "import random\n",
    "import json\n",
    "from collections import Counter\n",
    "from gensim.models import word2vec\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import logging"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 配置参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config():\n",
    "    #数据集路径\n",
    "    dataSource='./train.txt'\n",
    "    stopWords='./stopword.txt'\n",
    "    #分词后保留大于等于最低词频的词\n",
    "    minFreq=1\n",
    "    #统一输入文本序列的定长，超出的被阶段，不足的则补充0\n",
    "    sequenceLength=200\n",
    "    batchsize=64\n",
    "    epochs=10\n",
    "    \n",
    "    numsClass=2\n",
    "    rate=0.75\n",
    "    #生成嵌入词向量的维度\n",
    "    \n",
    "    embeddingdim=150\n",
    "    filterSize=[2,3,4,5]\n",
    "    keepProb=0.5\n",
    "    l2Lambda=0.1\n",
    "config=Config()  #实例化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 预训练词向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#设置输出日志\n",
    "logging.basicConfig(format=\"%(asctime)s:%(levelname)s:%(message)s\",level=logging.INFO)\n",
    "content=[]\n",
    "with open(config.dataSource) as f:\n",
    "    for line in f.readlines():\n",
    "        line=line.split('\\t')\n",
    "        content.append(jieba.cut(line[1]))\n",
    "        label.append(line[0])\n",
    "model=word2vec.Word2Vec(content,size=config.embeddingdim,min_count=config.minFreq,window=5,iter=20)\n",
    "moded.save(./word2vecModel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据预处理的类，生成训练集和测试集\n",
    "class Dataset(object):\n",
    "    def __init__(self, config):\n",
    "        self.dataSource = config.dataSource\n",
    "        self.stopWordSource = config.stopWordSource  \n",
    "        \n",
    "        # 每条输入的序列处理为定长\n",
    "        self.sequenceLength = config.sequenceLength  \n",
    "        self.embeddingSize = config.embeddingSize\n",
    "        \n",
    "        self.rate = config.rate\n",
    "        self.miniFreq=config.miniFreq\n",
    "        \n",
    "        self.stopWordDict = {}\n",
    "        \n",
    "        self.trainReviews = []\n",
    "        self.trainLabels = []\n",
    "        \n",
    "        self.evalReviews = []\n",
    "        self.evalLabels = []\n",
    "        \n",
    "        self.wordEmbedding =None\n",
    "        self.n_symbols=0\n",
    "        \n",
    "        self.wordToIndex = {}\n",
    "        self.indexToWord = {}\n",
    "        \n",
    "        \n",
    "        \n",
    "    def readData(self, filePath):\n",
    "        file = open(filePath) \n",
    "        text=[]\n",
    "        label=[]\n",
    "        for line in file:\n",
    "            temp=line.replace('\\n','').split(',,')\n",
    "            text.append(temp[0])\n",
    "            label.append(temp[1])\n",
    "        file.close()\n",
    "        \n",
    "        print('data:',len(text),len(label))\n",
    "        texts = [jieba.lcut(document.replace('\\n', '')) for document in text]\n",
    "\n",
    "        return texts, label\n",
    "    def readStopWord(self, stopWordPath):\n",
    "        \"\"\"\n",
    "        读取停用词\n",
    "        \"\"\"\n",
    "        \n",
    "        with open(stopWordPath, \"r\") as f:\n",
    "            stopWords = f.read()\n",
    "            stopWordList = stopWords.splitlines()\n",
    "            # 将停用词用列表的形式生成，之后查找停用词时会比较快\n",
    "            self.stopWordDict = dict(zip(stopWordList, list(range(len(stopWordList)))))\n",
    "    \n",
    "    \n",
    "    def getWordEmbedding(self, words):\n",
    "        \"\"\"\n",
    "        按照我们的数据集中的单词取出预训练好的word2vec中的词向量\n",
    "        \"\"\"\n",
    "        \n",
    "        #中文\n",
    "        model = gensim.models.Word2Vec.load('../data/word2VecModel')\n",
    "        \n",
    "        vocab = []\n",
    "        wordEmbedding = []\n",
    "        \n",
    "        # 添加 \"pad\" 和 \"UNK\", \n",
    "        vocab.append(\"pad\")\n",
    "        wordEmbedding.append(np.zeros(self.embeddingSize))\n",
    "        \n",
    "        vocab.append(\"UNK\")\n",
    "        wordEmbedding.append(np.random.randn(self.embeddingSize))\n",
    "        \n",
    "        for word in words:\n",
    "            try:\n",
    "                vector =model[word]\n",
    "                vocab.append(word)\n",
    "                wordEmbedding.append(vector)\n",
    "                \n",
    "            except:\n",
    "                print(word + \"不存在于词向量中\")\n",
    "                \n",
    "        return vocab, np.array(wordEmbedding)\n",
    "    def genVocabulary(self, reviews):\n",
    "        \"\"\"\n",
    "        生成词向量和词汇-索引映射字典，可以用全数据集\n",
    "        \"\"\"\n",
    "        \n",
    "        allWords = [word for review in reviews for word in review]\n",
    "        \n",
    "        #去掉停用词\n",
    "        subWords = [word for word in allWords if word not in self.stopWordDict]\n",
    "        \n",
    "        #统计词频，排序\n",
    "        wordCount = Counter(subWords)  \n",
    "        sortWordCount = sorted(wordCount.items(), key=lambda x: x[1], reverse=True)\n",
    "        \n",
    "        #去除低频词\n",
    "        words = [item[0] for item in sortWordCount if item[1] >= self.miniFreq ]\n",
    "        \n",
    "        \n",
    "        #获取词列表和顺序对应的预训练权重矩阵\n",
    "        vocab, wordEmbedding = self.getWordEmbedding(words)\n",
    "        \n",
    "        self.wordEmbedding = wordEmbedding\n",
    "        \n",
    "        self.wordToIndex = dict(zip(vocab, list(range(len(vocab)))))\n",
    "        self.indexToWord = dict(zip(list(range(len(vocab))), vocab))\n",
    "        self.n_symbols = len(self.wordToIndex) + 1\n",
    "        \n",
    "        # 将词汇-索引映射表保存为json数据，之后做inference时直接加载来处理数据\n",
    "        with open(\"../data/wordJson/wordToIndex.json\", \"w\", encoding=\"utf-8\") as f:\n",
    "            json.dump(self.wordToIndex, f)\n",
    "        \n",
    "        with open(\"../data/wordJson/indexToWord.json\", \"w\", encoding=\"utf-8\") as f:\n",
    "            json.dump(self.indexToWord, f)\n",
    "    def reviewProcess(self, review, sequenceLength, wordToIndex):\n",
    "        \"\"\"\n",
    "        将数据集中的每条评论里面的词，根据词表，映射为index表示\n",
    "        每条评论 用index组成的定长数组来表示\n",
    "        \n",
    "        \"\"\"\n",
    "        \n",
    "        reviewVec = np.zeros((sequenceLength))\n",
    "        sequenceLen = sequenceLength\n",
    "        \n",
    "        # 判断当前的序列是否小于定义的固定序列长度\n",
    "        if len(review) < sequenceLength:\n",
    "            sequenceLen = len(review)\n",
    "            \n",
    "        for i in range(sequenceLen):\n",
    "            if review[i] in wordToIndex:\n",
    "                reviewVec[i] = wordToIndex[review[i]]\n",
    "            else:\n",
    "                reviewVec[i] = wordToIndex[\"UNK\"]\n",
    "\n",
    "        return reviewVec\n",
    "\n",
    "    \n",
    "    \n",
    "    def genTrainEvalData(self, x, y, rate):\n",
    "        \"\"\"\n",
    "        生成训练集和验证集\n",
    "        \"\"\"\n",
    "        \n",
    "        reviews = []\n",
    "        labels = []\n",
    "        \n",
    "        # 遍历所有的文本，将文本中的词转换成index表示\n",
    "        for i in range(len(x)):\n",
    "            \n",
    "            reviewVec = self.reviewProcess(x[i], self.sequenceLength, self.wordToIndex)\n",
    "            reviews.append(reviewVec)\n",
    "            \n",
    "            labels.append([y[i]])\n",
    "            \n",
    "        trainIndex = int(len(x) * rate)\n",
    "        \n",
    "       \n",
    "        #trainReviews = sequence.pad_sequences(reviews[:trainIndex], maxlen=self.sequenceLength)\n",
    "        trainReviews = np.asarray(reviews[:trainIndex], dtype=\"int64\")\n",
    "        trainLabels = np.array(labels[:trainIndex], dtype=\"float32\")\n",
    "        \n",
    "        #evalReviews = sequence.pad_sequences(reviews[trainIndex:], maxlen=self.sequenceLength)\n",
    "        evalReviews = np.asarray(reviews[trainIndex:], dtype=\"int64\")\n",
    "        evalLabels = np.array(labels[trainIndex:], dtype=\"float32\")\n",
    "\n",
    "        return trainReviews, trainLabels, evalReviews, evalLabels\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    def dataGen(self):\n",
    "        \"\"\"\n",
    "        初始化训练集和验证集\n",
    "        \"\"\"\n",
    "        \n",
    "        #读取停用词\n",
    "        self.readStopWord(self.stopWordSource)\n",
    "        \n",
    "        #读取数据集\n",
    "        reviews, labels = self.readData(self.dataSource)\n",
    "        \n",
    "        #分词、去停用词\n",
    "        #生成 词汇-索引 映射表和预训练权重矩阵，并保存\n",
    "        self.genVocabulary(reviews)\n",
    "        \n",
    "        \n",
    "        #初始化训练集和测试集\n",
    "        trainReviews, trainLabels, evalReviews, evalLabels = self.genTrainEvalData(reviews, labels, self.rate)\n",
    "        self.trainReviews = trainReviews\n",
    "        self.trainLabels = trainLabels\n",
    "        \n",
    "        self.evalReviews = evalReviews\n",
    "        self.evalLabels = evalLabels\n",
    "data = Dataset(config)\n",
    "data.dataGen()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"train data shape: {}\".format(data.trainReviews.shape))\n",
    "print(\"train label shape: {}\".format(data.trainLabels.shape))\n",
    "print(\"eval data shape: {}\".format(data.evalReviews.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义网络结构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convolution(config):\n",
    "    sequence_length=config.sequenceLength\n",
    "    embedding_dimension=config.embeddingSize\n",
    "    \n",
    "    inn = Input(shape=(sequence_length, embedding_dimension, 1))\n",
    "    cnns = []\n",
    "    filter_sizes=config.filterSizes\n",
    "    for size in filter_sizes:\n",
    "        conv = Conv2D(filters=config.numFilters, kernel_size=(size, embedding_dimension),\n",
    "                            strides=1, padding='valid', activation='relu')(inn)\n",
    "        pool = MaxPool2D(pool_size=(sequence_length-size+1, 1), padding='valid')(conv)\n",
    "        cnns.append(pool)\n",
    "    outt =concatenate(cnns)\n",
    "\n",
    "    model = Model(inputs=inn, outputs=outt)\n",
    "    return model\n",
    "\n",
    "\n",
    "\n",
    "def cnn_mulfilter(n_symbols,embedding_weights,config):\n",
    "\n",
    "    model =Sequential([\n",
    "        \n",
    "        Embedding(input_dim=n_symbols, output_dim=config.embeddingSize,\n",
    "                        weights=[embedding_weights],\n",
    "                        input_length=config.sequenceLength),\n",
    "        \n",
    "        \n",
    "        Reshape((config.sequenceLength, config.embeddingSize, 1)),\n",
    "        \n",
    "        convolution(config),\n",
    "        Flatten(),\n",
    "        Dense(10, activation='relu',kernel_regularizer=regularizers.l2(config.l2RegLambda)),\n",
    "        Dropout(config.dropoutKeepProb),\n",
    "        Dense(1, activation='sigmoid')])\n",
    "        \n",
    "    model.compile(optimizer=optimizers.Adam(),\n",
    "                 loss=losses.BinaryCrossentropy(),\n",
    "                 metrics=['accuracy'])\n",
    "    return model\n",
    "    \n",
    "wordEmbedding = data.wordEmbedding\n",
    "n_symbols=data.n_symbols\n",
    "model = cnn_mulfilter(n_symbols,wordEmbedding,config)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = data.trainReviews\n",
    "y_train = data.trainLabels\n",
    "x_eval = data.evalReviews\n",
    "y_eval = data.evalLabels\n",
    "\n",
    "wordEmbedding = data.wordEmbedding\n",
    "n_symbols=data.n_symbols\n",
    "\n",
    "\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=10, mode='auto')\n",
    "\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5)\n",
    "model_checkpoint = ModelCheckpoint('./model/best_model/model_{epoch:02d}-{val_accuracy:.2f}.hdf5', save_best_only=True, save_weights_only=True)\n",
    "history = model.fit(x_train, y_train, batch_size=config.batchSize, epochs=config.epochs, validation_split=0.3,shuffle=True, callbacks=[reduce_lr,early_stopping,model_checkpoint])\n",
    "#验证\n",
    "\n",
    "scores = model.evaluate(x_eval, y_eval)\n",
    "\n",
    "#保存模型\n",
    "yaml_string = model.to_yaml()\n",
    "with open('./model/textCNN.yml', 'w') as outfile:\n",
    "    outfile.write( yaml.dump(yaml_string, default_flow_style=True) )\n",
    "model.save_weights('./model/textCNN.h5')\n",
    "\n",
    "print('test_loss: %f, accuracy: %f' % (scores[0], scores[1]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "history_dict = history.history\n",
    "history_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "acc = history.history['accuracy']\n",
    "val_acc = history.history['val_accuracy']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "epochs = range(1, len(acc) + 1)\n",
    "\n",
    "# \"bo\" is for \"blue dot\"\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "# b is for \"solid blue line\"\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
